#ifndef snap_memes_h
#define snap_memes_h

#if LOCAL
#include "../snap/Snap.h"
#else 
#include "/u/gxj/snap/libs/snap/Snap.h"
#endif

// qtInClust: Rember of a cluster, qtRoot:Root of a cluster, qtCentr: Artificial quote (cluster summary)
typedef enum {
  qtUndef, qtQuote, qtInClust, qtRoot, qtCentr
} TQtTy;

// url type
typedef enum {
  utUndef=0, utBlogLeft=1, utBlogRight=2, utBlog=3, utMedia=4
} TUrlTy; // utBlog is the default type

class TQuoteBs;
typedef TPt<TQuoteBs> PQuoteBs;
class TClustNet;
typedef TPt<TClustNet>  PClustNet;
typedef TNodeEDatNet<TPair<TStr, TInt>, TInt> TQtDomNet;  // domain name, url type
typedef TPt<TQtDomNet> PQtDomNet;

/////////////////////////////////////////////////
// Quote
#pragma pack(push, 1)
class TQuote {
public:
  class TTmUrlCnt;
  typedef TVec<TTmUrlCnt> TTmUrlCntV;
  typedef TVec<TPair<TSecTm, TFlt> > TTmFltPrV;
public:
  class TTmUrlCnt {
  private:
    TSecTm QtTm;
    int QtUrlId;
    uchar QtCnt;
  public:
    TTmUrlCnt() : QtTm(), QtUrlId(0), QtCnt(0) { }
    TTmUrlCnt(const TSecTm& Tm, const int& Url, const int& _QtCnt) : QtTm(Tm), QtUrlId(Url) {
      if (_QtCnt>255) { QtCnt=255; } else { QtCnt=_QtCnt; } }
    TTmUrlCnt(const TTmUrlCnt& UrlCnt) { memcpy((void*)this, (void*)&UrlCnt, sizeof(TTmUrlCnt)); }
    TTmUrlCnt(TSIn& SIn) { SIn.LoadBf(this, sizeof(TTmUrlCnt)); }
    void Load(TSIn& SIn) { SIn.LoadBf(this, sizeof(TTmUrlCnt)); }
    void Save(TSOut& SOut) const { SOut.SaveBf(this, sizeof(TTmUrlCnt)); }
    TSecTm Tm() const { return QtTm; }
    int UrlId() const { return QtUrlId; }
    void SetUrlId(const int& _UrlId) { QtUrlId=_UrlId; }
    int Cnt() const { return (int)QtCnt; }
    TTmUrlCnt& operator = (const TTmUrlCnt& Tuc) { if(this!=&Tuc){memcpy((void*)this, (void*)&Tuc, sizeof(TTmUrlCnt));} return *this; }
    bool operator == (const TTmUrlCnt& Val) const { return memcmp((void*) this, (void*)&Val, sizeof(TTmUrlCnt))==0; }
    bool operator < (const TTmUrlCnt& Val) const {
      return QtTm < Val.QtTm || (QtTm==Val.QtTm && UrlId()<Val.UrlId()) || (QtTm==Val.QtTm && UrlId()==Val.UrlId() && Cnt()<Val.Cnt()); }
  };
  class TQtIdTy {
  public:
    uint Id:26;
    uint Ty:4;
  public:
    TQtIdTy(const int& _Id=0, const TQtTy& _Ty=qtUndef) : Id(_Id), Ty((uint)_Ty) { IAssert(_Id<(int)TMath::Pow2(26) && int(_Ty)<16); }
    TQtIdTy(TSIn& SIn) { SIn.LoadBf(this, sizeof(TQtIdTy)); }
    void Load(TSIn& SIn) { SIn.LoadBf(this, sizeof(TQtIdTy)); }
    void Save(TSOut& SOut) const { SOut.SaveBf(this, sizeof(TQtIdTy)); }
  };
public:
  TQtIdTy QtCIdTy; // if type qtQuote: quote id; if qtInClust: id of the parent; qtRoot, qtCentr
  TStr QtStr;      // artificial cluster root quote has string QtStr
  TSecTm TmMed;	     // the average post time of the quote
  TFlt TmDev;	     // the time deviation of the post quote 
  TVec<TTmUrlCnt> TmUrlCntV;
public:
  TQuote() : TmMed(0), TmDev(-1) { }
  TQuote(TSIn& SIn) : QtCIdTy(SIn), QtStr(SIn), TmMed(SIn), TmDev(SIn), TmUrlCntV(SIn) { }
  void Save(TSOut& SOut) const { QtCIdTy.Save(SOut); QtStr.Save(SOut); 
    TmMed.Save(SOut); TmDev.Save(SOut); TmUrlCntV.Save(SOut); }
  void Load(TSIn& SIn) { QtCIdTy.Load(SIn); QtStr.Load(SIn);  
    TmMed.Load(SIn); TmDev.Load(SIn); TmUrlCntV.Load(SIn); }
  PXmlTok GetXmlTok() const { Fail; return TXmlTok::New("NodeData"); }

  int GetId() const { return QtCIdTy.Id; }
  int GetCId() const { return QtCIdTy.Id; }
  TQtTy GetTy() const { return (TQtTy) QtCIdTy.Ty; }
  TStr GetStr() const { return QtStr; }
  TSecTm GetTmMed() {if (TmMed.GetAbsSecs() <= 0) { ComputeTmStats(); } return TmMed;}
  TFlt GetTmDev() {if (TmDev < 0) { ComputeTmStats(); } return TmDev;}
  int GetFq() const; // actually returns the number of URLS
  int GetFq(const TSecTm& BegTm, const TSecTm& EndTm) const; // actually returns the number of URLS
  int GetUrls() const { return TmUrlCntV.Len(); }
  int GetDoms(const TQuoteBs& QtBs) const;
  int GetFq(const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  int GetFq(const TSecTm& BegTm, const TSecTm& EndTm, const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  int GetUrls(const TSecTm& BegTm, const TSecTm& EndTm, const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  int GetTimes() const { return TmUrlCntV.Len(); }
  TSecTm GetTm(const int& t) const { return TmUrlCntV[t].Tm(); }
  int GetUrlId(const int& t) const { return TmUrlCntV[t].UrlId(); }
  int GetCnt(const int& t) const { return TmUrlCntV[t].Cnt(); }
  TSecTm GetMinTm() const { return TmUrlCntV.Empty()?TSecTm():TmUrlCntV[0].Tm(); }
  TSecTm GetMaxTm() const { return TmUrlCntV.Empty()?TSecTm():TmUrlCntV.Last().Tm(); }
  TSecTm GetPeakTm(const TTmUnit& TmUnit=tmuDay, const TSecTm& AfterTm=TSecTm(1)) const;
  TSecTm GetPeakTm(const TTmUnit& TmUnit, const TSecTm& AfterTm, TInt& FreqAtPeak) const;
  TSecTm GetPeakTm(const TTmUnit& TmUnit, const TSecTm& AfterTm, const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  TSecTm GetMeanTm(const TTmUnit& TmUnit, const TUrlTy& UrlTy, const TQuoteBs& QtBs, const TSecTm& AfterTm=TSecTm(1)) const;
  TSecTm GetMedianTm(const TTmUnit& TmUnit, const TUrlTy& UrlTy, const TQuoteBs& QtBs, const TSecTm& AfterTm=TSecTm(1)) const;
  bool IsSinglePeak(const TTmUnit& TmUnit, const TSecTm& AfterTm, const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  TStr GetDesc() const;
  void ComputeTmStats();

  void GetFqOt(TTmFltPrV& RawFqOtV, const TTmUnit& TmUnit) const;
  void GetFqOt(TTmFltPrV& RawFqOtV, const TTmUnit& TmUnit, const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  void GetFqOt(TTmFltPrV& RawFqOtV, const TTmUnit& TmUnit, const TSecTm& BegTm, const TSecTm& EndTm) const;
  void GetFqOt(TTmFltPrV& RawFqOtV, const TTmUnit& TmUnit, const TSecTm& BegTm, const TSecTm& EndTm, const TUrlTy& UrlTy, const TQuoteBs& QtBs) const;
  void GetSmoothFqOt(TTmFltPrV& FqOtV, const TTmUnit& TmUnit, const int& WndSz, const double& Smooth,
    const TSecTm& BegTm=TSecTm(0), const TSecTm& EndTm=TSecTm(TInt::Mx-1)) const;
  void PlotOverTm(const TStr& OutFNm);

  static void GetSmoothFqOt(TTmFltPrV& FqOtV, const TTmFltPrV& RawFqOtV, const TTmUnit& TmUnit, const int& WndSz, const double& Smooth, const TSecTm& BegTm=TSecTm(0), const TSecTm& EndTm=TSecTm(TInt::Mx-1));
  static void LoadQtV(const TStr& InFNm, TVec<TQuote>& QtV);
  friend class TQuoteBs;
};
#pragma pack(pop)

typedef TVec<TQuote> TQuoteV;

/////////////////////////////////////////////////
// Quote Base
class TQuoteBs {
private:
  TCRef CRef;
  TStrHash<TInt, TBigStrPool> StrQtIdH;    // new for QtBs from Aug1 to Dec 31 (NEW VERSION, BIG POOL)
  THash<TInt, TQuote> QuoteH; // QtIds go from 0...
  TIntH UrlInDegH;  // Url/Domain in-degree (only for Domains/URLs that have quotes)
	THash<TInt, TIntSet> UrlLkH;	//Url link hashset
  TIntH UrlTyH;     // UrlId to TypeId (only for known URLs that have quotes)
  THash<TInt, TIntV> ClustQtIdVH; // root quote id --> all quotes in the cluster
  TSecTm MnTm;
  TSecTm MxTm;
public:
  TQuoteBs() : StrQtIdH() { } // reserve big pool: StrQtIdH(TBigStrPool::New(Mega(2000), Mega(256))) 
  TQuoteBs(TSIn& SIn);
  void Save(TSOut& SOut) const;
  static PQuoteBs New();
  static PQuoteBs Load(TSIn& SIn);

  int GetStrs() const { return StrQtIdH.Len(); }
  const char *GetStr(const int& StrId) const { return StrQtIdH.GetKey(StrId); }
  bool IsStrId(const int& StrId) const { return StrId < StrQtIdH.Len(); }
  bool IsStr(const char* CStr) const { return StrQtIdH.IsKey(CStr); }
  bool IsStr(const TChA& Str) const { return StrQtIdH.IsKey(Str.CStr()); }
  int GetStrId(const char* CStr) const { return StrQtIdH.GetKeyId(CStr); }
  int GetStrId(const TChA& Str) const { return StrQtIdH.GetKeyId(Str.CStr()); }
  int AddStr(const char* CStr) { return StrQtIdH.AddDat(CStr, -1); }
  int AddStr(const TChA& ChA) { return StrQtIdH.AddDat(ChA.CStr(), -1); }

  int Len() const { return QuoteH.Len(); }
  bool IsQt(const char* CStr) const { return GetQtId(CStr)!=-1; }
  bool IsQtId(const int& QtId) const { return QuoteH.IsKey(QtId); }
  int GetQtId(const int& QtN) const { return QuoteH.GetKey(QtN); }
  int GetQtId(const char* CStr) const { return IsStr(CStr)? StrQtIdH.GetDat(CStr).Val : -1; }
  const TQuote& GetQt(const int& QtId) const { return QuoteH.GetDat(QtId); }
  TQuote& GetQt(const int& QtId) { return QuoteH.GetDat(QtId); }
  const TQuote& GetQtN(const int& QtN) const { return QuoteH[QtN]; }
  TQuote& GetQtN(const int& QtN) { return QuoteH[QtN]; }
  void DelQtId(const int& QtId) { return QuoteH.DelKey(QtId); }

  TUrlTy GetUrlTy(const int& UrlId) const;
  void SetUrlTy(const TStr& InFNm, const TUrlTy& SetTy);

  void GetQtIdV(TIntV& QtIdV) const;
  void GetQtIdVByFq(TIntV& QtIdV, const int& MinWrdLen, const int& MinQtFq, const bool& OnlyClustRoots=true,
    const TStr& HasWord="", const TStr& AppearsAtUrl="", const TUrlTy& OnlyCountTy=utUndef, const TSecTm& BegTm=TSecTm(0), const TSecTm& EndTm=TSecTm(TInt::Mx-1)) const;
  void GetQtIdVByFq(TIntV& QtIdV, const int& MinWrdLen, const int& MinQtFq, const TStrV& FromDomains, const bool& OnlyClustRoots=true, const TStr& HasWord="", int MinDoms=5) const;
  void GetQtIdVByTm(const int& WndSzHr, const int& StepHr, const int& MinWrdLen, const int& MinQtFq, const int& TakePerStep) const;
  void GetCIdVByFq(TIntV& CIdV, const int& MinClFq, const TStr& RootHasWord="", const TUrlTy& OnlyCountTy=utUndef,
    const bool& OnlyAfterRoot=false, const TSecTm& BegTm=TSecTm(0), const TSecTm& EndTm=TSecTm(TInt::Mx-1)) const;
  void GetMinMaxTm(TSecTm& MinTm, TSecTm&MaxTm) const;
  void GetMinMaxTm(const TIntV& QtIdV, TSecTm& MinTm, TSecTm&MaxTm) const;
  void GetQtPageUrl(const TIntV& QtIdV, TIntH& QtUrlIdH) const;

  void AddQuote(const TQuote& Quote, const TQuoteBs& CurQtBs);
  void AddQuote(const TVec<TChA>& QuoteV, const TVec<TChA>& LinkV, const TChA& PostUrlStr, const TSecTm& PubTm, const int& MinQtWrdLen=4);
  PQuoteBs GetQuoteBs(const TIntV& QtIdV) const;

  int GetClusts() const { return ClustQtIdVH.Len(); }
  int GetCId(const int& ClustN) const { return ClustQtIdVH.GetKey(ClustN); }
  bool IsClust(const int& CId) const { return ClustQtIdVH.IsKey(CId); }
  const TIntV& GetClust(const int& CId) const { return ClustQtIdVH.GetDat(CId); } // CId == RootQtId
  const TIntV& GetClustN(const int& N) const { return ClustQtIdVH[N]; } // CId == RootQtId
  int GetQtsInClust() const;
  int GetClustFq(const int& CId) const;
  int GetClustFq(const int& CId, const TUrlTy& UrlTy) const;
  int GetClustMxQtFq(const int& CId) const;
  int GetCentrQtId(const int& CId) const;
  int GetCentrQtId(const TIntV& QtIdV) const;
  void GetMergedClustQt(const int& CId, TQuote& NewQt, const bool& OnlyAfterBegTm=true) const;
  void GetMergedClustQt(const TIntV& QtIdV, TQuote& NewQt, const bool& OnlyAfterBegTm=true) const;
  TSecTm GetClustBegTm(const int& CId, const int& CentrQtId) const;
  TSecTm GetClustBegTm(const TIntV& ClustV, const int& CentrQtId) const;
  void CreateClusters(const TVec<TIntV>& ClustQtIdV);
  void ClusterQts(const int& MinRootWrdLen, const int& MinQtFq, const TStr& OutFNmPref, const TStrV& BlackListV=TStrV());
  void ResetClusters();
  void ReassignToClust(const int& QtId, const int& NewCId);
  void Mergec2Clusters(const int& ParentCId, const int& ChildCId);
  PClustNet GetClustNet(const int& MinQtFq, const TStr& OutFNmPref) const;
  PQuoteBs GetMergeClusters(const bool& OnlyClusters=true) const;
  void AddMergedQtsToQtBs();
	void GetSampleQtBs(int MinFq, TStr Pref);
	void GetSampleQtBs(TIntV& QtV, TStr Pref);

  void GetTopQtDoms(TStrIntPrV& DomCntV, const int& TakeNClust, const int& MinClFq, const TStr& RootHasWord="", const TUrlTy& OnlyCountTy=utUndef) const;
  void TopDomsByLag(const TTmUnit& TmUnit, const int& TakeNDoms, const int& TakeNClusts, const int& Thresh) const;

  void PlotQtFqCnt(const TStr& OutFNmPref) const;
  void PlotQtMediaVsBlogFq(const int& QtId, const TStr& OutFNmPref) const;
  void PlotClustMediaVsBlogFq(const int& CId, const TStr& OutFNmPref) const;
  void PlotMediaVsBlogLag(const TTmUnit& TmUnit, const bool& TakeClusters, const int& TakeN, const TStr& OutFNmPref) const;
  void PlotFqDecay(const TTmUnit& TmUnit, const bool& TakeClusters, const TUrlTy& CntUrlTy, const int& PlotN, const int& MinValsPerTm, const TStr& OutFNmPref) const;
  void PlotBlogVsMediaFqOt(const TTmUnit& TmUnit, const bool& TakeClusters, int PlotN, const TStr& OutFNmPref) const;
  void PlotBlogFracOt(const TTmUnit& TmUnit, const bool& TakeClusters, const int& PlotN, const TStr& OutFNmPref) const;
  void PlotPopularityCnt(const bool& TakeClusters, const TUrlTy& UrlTy, const int& PlotN, const TStr& OutFNmPref) const;
  void PlotEmptyY(const TTmUnit& TmUnit, const bool& TakeClusters, const int& PlotN, const TStr& OutFNmPref) const;

  PNGraph GetQuotePostNet(const TStr& DatasetFNm) const;
  PQtDomNet GetQuoteDomNet(const PNGraph& PostLinkGraph, const int& CId) const;

  void SaveQuotes(const int& MinQtFq, const TStr& OutFNm) const;
  void SaveQuotes(const TIntV& QtIdV, const TStr& OutFNm) const;
  void SaveClusters(const TStr& OutFNm, const bool& SkipUrls=true) const;
  void SaveClusters(const TIntV& ClustIdV, const TStr& OutFNm, const bool& SkipUrls=true) const;
	void AddMergedClusters();
  void SaveMergedClusters(const TStr& OutFNm); 
  void SaveBigBlogMassQt(const TStr& OutFNm) const;
  void SaveForFlash(const TIntV& QtIdV, const TStr& OutFNm, const TTmUnit& TmUnit, int SaveN, const TSecTm& BegTm=TSecTm(1), const TSecTm& EndTm=TSecTm(TInt::Mx-1), const TUrlTy& OnlyCountTy=utUndef) const;
  void SaveForFlash2(const TIntV& QtIdV, const TStr& OutFNm, const TTmUnit& TmUnit, int SaveN, const TSecTm& BegTm=TSecTm(1), const TSecTm& EndTm=TSecTm(TInt::Mx-1), const TUrlTy& OnlyCountTy=utUndef) const;
  void SaveForFlashAll(const TIntV& QtIdV, const TStr& OutFNm, const TTmUnit& TmUnit, const TSecTm& BegTm=TSecTm(1), const TSecTm& EndTm=TSecTm(TInt::Mx-1), const TUrlTy& OnlyCountTy=utUndef) const;
  void SaveDomainStat(const TStr& OutFNm, const int& MinCnt) const;
  void Dump(const bool& Fast=false) const;
	void DumpClusterQuoteBs(TStr Pref);

  // Quote clustering
	static int LongestCmnSubSq(const TIntV& WIdV1, const TIntV& WIdV2, int& WIdV1Start, int& WIdV2Start, int& SkipId);
  void ShowPathInfo(int CId, int srcQt, int dstQt, PClustNet& Net, TStr& PathFNm);
  void RegisterTime(TStr Desc, bool Start, TStrHash<TStr>& TmRecH);
  void ShowAllTime(TStrHash<TStr>& TmRecH);
  bool IsLinkPhrases(const int& QtN1, const int& QtN2, THash<TInt, TIntV>& QtToWordIdVH);
  bool IsLinkPhrases2(const int& QtN1, const int& QtN2, THash<TInt, TIntV>& QtToWordIdVH);
  void HashingShgls(int MinQtFq, int MnWrdLen, int MxWrdLen, THash<TMd5Sig, TIntV>& ShglQtIdVH, THash<TInt, TIntV>& QtToWordIdVH, TStrHash<TInt>& WordIdH);
  void ElimBadBkt(TStr& BlackListFNm, double BktThresh, THash<TMd5Sig, TIntV>& ShglQtIdVH, THash<TInt, TIntV>& QtToWordIdVH);
  void BktAddLink(PClustNet& Net, THash<TMd5Sig, TIntV>& ShglQtIdVH, THash<TInt, TIntV>& QtToWordIdVH);
  void EvaluateLinks(PClustNet& Net, double MxTmDelay, double MxTmDev);
  void RefineCluster(PClustNet& Net, double RefineThresh, THash<TInt, TIntV>& QtToWordIdVH);
	void RefineQtBs(TStr& BlackListFNm, int MinQtFq, int MnWrdLen, int MxWrdLen);

  void ConstructQtBsZarya(TStr InFNm, TStr Pref, TStr MediaUrlFNm, TSecTm MinTm, TSecTm MaxTm, int MinWrdLen, int MinMemeFq, int HSize = 100, int UrlSize = 500);
	void ConstructQtBsQtOnly(TStr InFNm, TStr Pref, TStr MediaUrlFNm, int MinWrdLen, int MinMemeFq, int HSize);
	void ConstructQtBsQtTime(TStr InFNm, TStr Pref, TStr MediaUrlFNm, int MinWrdLen, int MinMemeFq, int HSize);
  void ClusterQuotes(int MinQtFq,int MnWrdLen, int MxWrdLen, TStr& BlackListFNm, TStr& Pref, bool IsShglReady, bool IsNetReady, double BktThresh, double MxTmDelay, double MxTmDev, double RefineThresh);
  void DumpQuoteClusters(int MinWrdLen, int MinClustFq, bool SkipUrl, bool FlashDisp, TStr Pref);
	void AnalyzeQuoteTrend(TStr Pref, const TTmUnit TmUnit = tmu1Hour, const int TmCnt = 10, int TopN = TInt::Mx);

//	// quote trend analysis
//	void GetDomGraph(TNodeEDatNet<TInt, TInt>& FBNet, TNodeEDatNet<TInt, TInt>& CoMDomNet, THash<TMd5Sig, TInt>& DomIdH, THash<TInt, TChA>& DomStrH, bool DoVariant = true);
////	void GetDomLinkGraph(TNodeEDatNet<TInt, TInt>& DomNet, THash<TMd5Sig, TInt>& DomIdH, THash<TInt, TChA>& DomStrH, bool DoVariant = true);
//	void GetMixDomGraph(TNodeEDatNet<TFlt, TFlt>& MixDomNet, const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& RawCoMDomNet);
//	void GetDomUNGraph(PUNGraph& DomUNGraph, const TNodeEDatNet<TInt, TInt>& DomNet, TIntH& CcH);
//	void RefineDomLinkGraph(TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& RawDomNet, THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH, TStr BlackListFNm);
//	void RefineDomCoMGraph(TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& RawDomNet);
//	void PrintDomNetInfo(const TNodeEDatNet<TInt, TInt>& DomNet);
//
//	void GetTrendQtPrV(TIntPrV& QtPrV, int MinUrls, int TopN, const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, 
//		const TTmUnit& TmUnit = tmu1Hour, int TmCnt = 6, bool DoVariant = true);	
//	void ComputeQtSig(int QtN, TQuote::TTmFltPrV& SmoothFqOtV, const TTmUnit& TmUnit, int TmCnt, TSecTm BegTime = TSecTm(0));
//	void ComputeQtSig(TQuote& Qt, TQuote::TTmFltPrV& SmoothFqOtV, const TTmUnit& TmUnit, int TmCnt, TSecTm BegTime = TSecTm(0));
//	void PlotTrendQtPrV(TIntPrV& QtPrV, TStr Pref, const TTmUnit& TmUnit, int TmCnt);
//	void PlotClusterPair(TStr& OutFNm, int QtN1, int QtN2, TQuote::TTmFltPrV& SigV1, TQuote::TTmFltPrV& SigV2, const TTmUnit& TmUnit);
//	void PlotTopClusterVariants(int TopNClust, int TopNVar, TStr Pref, const TTmUnit& TmUnit, int TmCnt);
//
//	void GetFeatureStrV(TStrV& StrV, TFltV& IntvlV);
//	void ComputeGraphFeatures(TFltV& FeatureV, const int QtN, TIntV& DomIdV, const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& CoMDomNet, const TNodeEDatNet<TInt, TInt>& RawCoMDomNet, 
//		const TNodeEDatNet<TFlt, TFlt>& MixDomNet, const PUNGraph& DomUNGraph, const TIntH& CcH, 
//		const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH, const THash<TInt, TFlt>& DomTmH, int MaxN = -1);
//	void DumpFeatureStats(TStr Pref, const TIntPrV& QtPrV, const TVec<TPair<TFltV, TFltV> >& QtPrFeatureV, TStrV& FeatureStrV, const TFltV IntvlV);
//
//	// feature computation
//	double ComputeQtEdgeNum(const TVec<TFltV>&DomAdj);
//	double ComputeQtTriadNum(const TVec<TFltV>&DomAdj);
//	void ComputeQtGcc(const TVec<TFltV>& DomAdj, double& compNum, double& gccNodeNum, double& gccEdgeNum);
//	void ComputeQtDeg(const TVec<TFltV>& DomAdj, double& maxDeg, double& d0NodeNum, double& d1NodeNum);	
//	void ComputeQtInOutDeg(const TNodeEDatNet<TInt, TInt>& DomNet, const TIntV& DomIdV, double& TotInDeg, double& TotOutDeg, double& AvgInDeg, double& AvgOutDeg, double& AvgInOutDegR, double& AvgOutInDegR);
//	double ComputeQtDensity(const TVec<TFltV>& DomAdj);
//	void ComputeDomScores(const TIntSet& InfDomIdSet, TIntFltH& ScoreH, const TNodeEDatNet<TInt, TInt>& DomNet);
//	double ComputeQtScore(const TIntSet& DomIdSet, const TNodeEDatNet<TInt, TInt>& DomNet);
//	double ComputeQtClusterCoeff(const TIntSet& DomIdSet, const TNodeEDatNet<TInt, TInt>& DomNet);
//	double ComputeQtClusterCoeff(const TVec<TFltV>& DomAdj);
//	double ComputeQtMBRatio(const TIntSet& DomIdSet, const THash<TInt, TChA>& DomStrH);
//	double ComputeQtReportTime(const TIntSet& DomIdSet, const THash<TInt, TFlt>& DomTmH);
//	double ComputeQtWordNum(const int QtN);
//	void ComputeQtExt(double& ExtNodeNum, double& ExtEdgeNum, const TNodeEDatNet<TInt, TInt>& DomNet, const TIntSet& DomIdSet, const TIntH& TriadH);
//	void ComputeQtMixDeg(double& MixInDeg, double& MixOutDeg, double& MixExtInDeg, double& MixExtOutDeg, const TIntSet& DomIdSet, const TNodeEDatNet<TFlt, TFlt>& MixDomNet);
//	double ComputeQtEntExcessDeg(const TVec<TFltV>& DomAdj);
//	void ComputeQtCompetitionCoeff(double& InCompCoeff, double& OutCompCoeff, const TIntSet& DomIdSet);
//
//	void ComputeAllQtReportTime(THash<TInt, TFlt>& DomTmH, const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH);
//	void ComputeTriadH(const TNodeEDatNet<TInt, TInt>& DomNet, TIntH& LinkTriadH);
//	void ComputeMixDegH(const TNodeEDatNet<TFlt,TFlt>& MixDomNet, TIntFltH& MixInDegH, TIntFltH& MixOutDegH, TIntFltH& MixExtInDegH, TIntFltH& MixExtOutDegH);
//	void ComputeCompH(TIntPrFltH& InCompH, TIntPrFltH& OutCompH, const TNodeEDatNet<TInt,TInt>& DomNet, const TNodeEDatNet<TInt,TInt>& RawCoMDomGraph);
//	void ComputeJaccardSim(double& InSim, double& OutSim, int SrcId, int DstId, const TNodeEDatNet<TInt,TInt>& DomNet);
//
//	// Utility function
//	void DumpClusterPairHTML(TStr& FN, int QtN1, int QtN2, const TTmUnit& TmUnit, const int& TmCnt);
//	void DumpTrendQt(TIntPrV& QtVarPrV, TStr OutFNm, bool SkipUrls = true);
//	void DumpDomains(TStr FNm, const TNodeEDatNet<TInt, TInt>& DomNet, THash<TMd5Sig, TInt>& DomIdH, THash<TInt, TChA>& DomStrH);
//	void ThreshDomains(double Thresh, TNodeEDatNet<TInt, TInt>& DomNet);
//	void GetQtDomIdSet(TIntSet& DomIdSet, int QtN, const TTmUnit& TmUnit, int TmCnt, const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, int MaxN = -1);
//	void GetQtDomIdV(TIntV& DomIdV, int QtN, const TTmUnit& TmUnit, int TmCnt, const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, int MaxN = -1);
//	void GetSubGraphFromDomNet(TVec<TFltV>& DomAdj, const TIntSet& DomIdSet, const TNodeEDatNet<TInt, TInt>& DomNet);
//	void GetSubGraphFromDomNet(TVec<TFltV>& DomAdj, const TIntSet& DomIdSet, const TNodeEDatNet<TFlt, TFlt>& DomNet);
//	void ComputeConnDomIdSet(TIntSet& ConnDomIdSet, const TIntV& DomIdV, const PUNGraph& DomUNGraph, const TIntH& GlobalCcH);
//	TStr GetDomNmFromUrl(TStr UrlStr);
//
//	// Machine learning
//	void SVMtoMATLAB(const TIntPrV& QtPrV, const THash<TInt, TIntV>& QtDomIdVH, TIntH QtLabelH);
//	void SVMtoMATLAB2(const TIntPrV& QtPrV, const THash<TInt, TIntV>& QtDomIdVH, TIntH QtLabelH);
//	void LogittoMATLAB(const THash<TInt, TFltV>& QtFeatureVH, const TIntH& QtLabelH);
//	void ComputeDecisionTree(const TVec<TPair<TFltV, TFltV> >& QtPrFeatureV, const TStrV& FeatureStrV);
//	
//	// Unsorted functions
//  void GetClustLkGraph(TNodeNet<TSecTm>& LnkG, const TQuote& ClustQt);
//	void Validate(int BegN, int EndN, const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH);
//	void ProcessEdge(const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH);	
//	void TimeIntvlStats(THash<TSecTm, TFlt>& TmStatsH, const TTmUnit& TmUnit);
//	void ComputeDuplicateDomain(TStr Pref, int MinFq, double MinSim, const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TInt, TChA>& DomStrH);
//	
//	// Test functions
//	void TESTGetDuplicateDoms(const TNodeEDatNet<TInt,TInt>& DomNet, const TNodeEDatNet<TInt,TInt>& RawCoMDomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH);
//	void TESTEdgeCausal(const TNodeEDatNet<TInt,TInt>& DomNet, const TNodeEDatNet<TInt,TInt>& RawCoMDomNet, const THash<TMd5Sig, TInt>& DomIdH);
//	void TESTDomainPrediction(const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH, TIntPrV& QtVarPrV, THash<TInt, TIntV>& QtDomIdVH);
//	void EvaluateFeatureQuality(const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& CoMDomNet, const TNodeEDatNet<TInt, TInt>& RawCoMDomNet, 
//		const TNodeEDatNet<TFlt, TFlt>& MixDomNet, const PUNGraph& DomUNGraph, TIntH& CcH, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH,
//	  const THash<TInt, TFlt>& DomTmH, const TTmUnit TmUnit, const int TmCnt, TStrV& FeatureStrV, const TFltV& IntvlV);
//	void TESTCoMentionLinkG(const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& CoMDomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH);
//	void TESTRewireCoMGraph(const TNodeEDatNet<TInt, TInt>& RawCoMDomNet, const THash<TMd5Sig, TInt>& DomIdH);
//	void TESTBadQuoteInvestigation(const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& CoMDomNet, const TNodeEDatNet<TInt, TInt>& RawCoMDomNet, 
//		const TNodeEDatNet<TFlt, TFlt>& MixDomNet, const PUNGraph& DomUNGraph, TIntH& CcH, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH, 
//		const THash<TInt, TFlt>& DomTmH, const TTmUnit TmUnit, const int TmCnt, TStrV& FeatureStrV, const TFltV& IntvlV);
//	void TESTFeatureEvolution(const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& CoMDomNet, const TNodeEDatNet<TInt, TInt>& RawCoMDomNet, 
//		const TNodeEDatNet<TFlt, TFlt>& MixDomNet, const PUNGraph& DomUNGraph, TIntH& CcH, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH,
//	  const THash<TInt, TFlt>& DomTmH, const TTmUnit TmUnit, const int TmCnt, TStrV& FeatureStrV, const TFltV& IntvlV);
//	void TESTComputeTotQtVolH();
//	
//	void TESTDiscriminationTask(const TIntPrV& QtPrV, const TNodeEDatNet<TInt, TInt>& DomNet, const TNodeEDatNet<TInt, TInt>& CoMDomNet, const TNodeEDatNet<TInt, TInt>& RawCoMDomNet, 
//		const TNodeEDatNet<TFlt, TFlt>& MixDomNet, const PUNGraph& DomUNGraph, TIntH& CcH, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH,
//	  const THash<TInt, TFlt>& DomTmH, const TTmUnit TmUnit, const int TmCnt, TStrV& FeatureStrV, const TFltV& IntvlV);
//	void TESTNaiveBayes(const TNodeEDatNet<TInt, TInt>& DomNet, const THash<TMd5Sig, TInt>& DomIdH, const THash<TInt, TChA>& DomStrH);
//
//	// HTML functions
//	void HTMLFeaturePlot(TStr Path, TStrV& FeatureStrV);
private:
	void ComputeMnMxTm();
	static void QuoteFilter(TStr &QtStr);
	static void GetFileList(TStr InFNm, TChAV& filelist);
	static bool IsValidQuote(const TQuote& Q, const int MnWrdLen, const int MxWrdLen = 200);
	static bool CleanURL(TChA &UrlStr);
	static double QuoteDistance(TInt Qt1, TInt Qt2, THash<TInt, TIntV>& QtToWordIdVH);
	static void AddElemPool(THashSet<TIntPr>& ElemPool, TIntV& QtV, const PQuoteBs QtBs, THash<TInt, TIntV>& QtToWordIdVH);
	static void AddElemPool(THashSet<TIntPr>& ElemPool, THashSet<TIntPr>& CurElemPool);
	static void GetTmFltDataPr(TFltPrV& Data, TQuote::TTmFltPrV& SigV, const TTmUnit& TmUnit);
	void FilterBlackListQuotes(TStr FNm);
  static PQuoteBs FindQtInTxtQBs(const TStr& InBinFNmWc, const TStrV& QuoteStrV);

public:
  friend class TPt<TQuoteBs>;
	friend class TQuote;
	friend class TMemeTrend;
};

/////////////////////////////////////////////////
// Cluster Network
class TClustNet : public TNodeNet<TQuote> {
public:
  TClustNet() { }
  TClustNet(TSIn& SIn) : TNet(SIn) { }
  void Save(TSOut& SOut) const { TNet::Save(SOut); }
  static PClustNet New() { return new TClustNet(); }
  static PClustNet Load(TSIn& SIn) { return new TClustNet(SIn); }

  void AddLink(const TQuote& SrcQt, const TQuote& DstQt);
  PClustNet GetSubGraph(const TIntV& NIdV) const;
  PClustNet GetSubGraph(const int& MinQtWords, const int& MaxQtWords, const int& MinFq) const;

  void RecalcEdges(const double& MinOverlapFrac);
  //void DelLongQts(const int& MxWrdLen);
  void MakeClusters(const TIntPrV& KeepEdgeV);
  void KeepOnlyTree(const TIntPrV& KeepEdgeV);
  void GetClusters(TVec<TIntV>& QtNIdV) const;
  void GetMergedClustQt(const TIntV& QtIdV, TQuote& NewQt) const;

  int EvalPhraseClusters(const TIntPrV& KeepEdgeV, const bool& dump=true) const;
  void ClustKeepSingleEdge(const int& MethodId) const;
  void ClustKeepSingleEdge(const int& MethodId, TIntPrV& KeepEdgeV) const;
  void ClustGreedyTopDown() const;
  void ClustGreedyTopDown(TIntPrV& KeepEdgeV) const;
  void ClustGreedyRandom() const;

  PClustNet GetThis() const { IAssert(CRef.GetRefs()>0); return PClustNet((TClustNet *) this); }
  void DrawNet(const TStr& OutFNm, const int& SaveTopN=10) const;
  void DumpNodes(const TStr& OutFNm, const int& SaveTopN=10) const;
  void DumpClusters(const TStr& OutFNm, int SaveTopN=10) const;
  void DumpClustersByVol(const TStr& OutFNm, const int& MinClustSz, const int& MinVolume) const;

	static TChA InsertLineBreaks(const TChA& ChA, const int& BreakAtPost);
  static void DumpTmpCluster(TStr Nm, THash<TInt, TIntV>& QtIdCcIdVH, const PQuoteBs& QtBs, PClustNet& Net);

  friend class TPt<TClustNet>;
};

/////////////////////////////////////////////////
// Quote Loader
class TQuoteLoader {
private:
  TFFile FFile;
  PSIn SIn;
  TExeTm ExeTm;
public:
  TStr CurFNm;
  int PostCnt;
  TSecTm PubTm;
  TChA PostUrlStr;
  TChA PostTitleStr;
  TChA BlogUrlStr;
  TChA BlogTitleStr;
  TChA ContentStr;
  TVec<TChA> QuoteV;
  TVec<TChA> LinkV;
private:
  bool LoadItem(TXmlLx& XmlLx);
public:
  TQuoteLoader(const TStr& InFNmWc) : FFile(InFNmWc), PostCnt(0) { }
  virtual ~TQuoteLoader() { }
  void Save(TSOut& SOut) const;
  void Load(TSIn& SIn);
  void Clr();
  bool Next();
  // events
  void ProcessPosts(const bool& IsXml=false, int LoadN=-1);
  virtual void StartProcess() { }
  virtual void EndProcess(const int& PostCnt) { }
  virtual void StartProcFile(const TStr& FNm) { }
  virtual void EndProcFile(const TStr& FNm) { }
  virtual void ProcessPost(const bool& GoodItem) { }
};

/////////////////////////////////////////////////
// Memes Dataset Loader
// FORMAT:
//U \t Post URL
//D \t Post time
//T \t Post title (optional!)
//C \t Post content
//L \t Index \t URL      (URL starts at Content[Index])
//Q \t Index \t Length \t Quote (Quote starts at Content[Index])
// EXAMPLE:
//U       http://twitter.com/celebritybuzzon/statuses/21622719142
//D       2010-08-19 17:59:42
//C       free energy release free "bang pop" remix mp3, release split 7 ": altsounds.com... and performing live at spin hou...
//L       118     http://bit.ly/aI5wOF
//Q       26      8       bang pop

class TMemesDataLoader {
private:
  PFFile FFile;
  PSIn InFNmF;
  PSIn SInPt;
  TChA CurLn;
  bool GetNextFile();
public:
  uint64 LineCnt;
  TChA PostUrlStr;
  TChA ContentStr;
  TSecTm PubTm;
  TVec<TChA> MemeV;       // quote
  TVec<TIntPr> MemePosV;  // (index, length), quote begins at ContentStr[MemePos[i]]
  TVec<TChA> LinkV;       // link url
  TVec<TInt> LinkPosV;    // url begins at ContentStr[LinkPosV[i]]
public:
  TMemesDataLoader() {}
  TMemesDataLoader(const TStr& FNmWc, const bool& IsWc=true) {
    if (IsWc) { FFile = TFFile::New(FNmWc, false); } // wild-card expression for loading a list of files
    else { InFNmF = TFIn::New(FNmWc); } } // file with one filename in each line
  void LoadFile(const TStr& FNm) {
	  if (TZipIn::IsZipExt(FNm.GetFExt())) { SInPt = TZipIn::New(FNm); }
	  else { SInPt = TFIn::New(FNm); }  LineCnt = 0; }
  void Clr();
  bool LoadNext();
	bool LoadNext4(THashSet<TMd5Sig>& ElemSet);
  bool LoadNextSkip();
  void SaveTxt(TSOut& SOut) const;
  void Dump(const bool& DumpAll=false) const;
};

class TmYMDH {
private:
	int year;
	int month;
	int date;
	int hour;
	static int EndDate[13];
public:
	TmYMDH() : year(0), month(0), date(0), hour(0){}
	TmYMDH(int y, int m, int d, int h) : year(y), month(m), date(d), hour(h) {}
	TmYMDH(TSecTm& Tm) : year(Tm.GetYearN()), month(Tm.GetMonthN()), date(Tm.GetDayN()), hour(Tm.GetHourN()) {};

	TmYMDH& operator=(const TmYMDH& Tm) {year = Tm.year;month = Tm.month;date = Tm.date;hour = Tm.hour;	return *this;}

	bool operator<(const TmYMDH& Tm) { return (year < Tm.year) || 
			(year == Tm.year && month < Tm.month) ||
			(year == Tm.year && month == Tm.month && date < Tm.date) ||
			(year == Tm.year && month == Tm.month && date == Tm.date && hour < Tm.hour);
	}

	TmYMDH& operator+=(const int& inc) {
		for (int i = 0; i < inc; i++) {
			hour++;
			if (hour == 24) {
				hour = 0; date++;
				EndDate[2] = GetFebDate(year);
				if (date > EndDate[month]) {
					date = 1; month++;
					if (month > 12) {
						month = 1; year++;
					}
				}
			}
		}
		return *this;
	}

	TStr GetStr() {
		TStr out, month0, date0, hour0;
		if (month < 10)	month0 = "0"; else month0 = "";
		if (date < 10) 	date0 = "0"; else date0 = "";
		if (hour < 10)	hour0 = "0"; else hour0 = "";
		out = TStr::Fmt("%d-%s%d-%s%dT%s%d-00-00Z", year, month0.CStr(), month, date0.CStr(), date, hour0.CStr(), hour);
		return out;
	}

	static void GenSpinn3rList(TmYMDH BegTm, TmYMDH EndTm, TStr Path, TStr OutFNm) {
		FILE* FOut = fopen(OutFNm.CStr(), "w");
		for (TmYMDH Tm = BegTm; Tm < EndTm; Tm += 1) {
			TmYMDH NextTm = Tm; NextTm += 1;
			fprintf(FOut, "%s%s_%s.rar\n", Path.CStr(), Tm.GetStr().CStr(), NextTm.GetStr().CStr());
		}
		fclose(FOut);
	}

private: 
	int GetFebDate(int year) {
		if (year % 400 == 0) return 29;
		if (year % 100 == 0) return 28;
		if (year % 4 == 0) return 29;
		return 28;
	}
};

class TMemeTrend {
private:
	PQuoteBs QtBs;
	TStr Pref;

	THash<TMd5Sig, TInt> DomIdH;
	THash<TInt, TChA> DomStrH;

	TNodeEDatNet<TInt, TInt> RawLinkDomNet, RawCoMDomNet;
	TNodeEDatNet<TInt, TInt> LinkDomNet, CoMDomNet;
	TNodeEDatNet<TFlt, TFlt> MixDomNet;
	PUNGraph DomUNGraph;
	TIntH CcH;
	TStrV FeatureStrV;
	TFltV IntvlV;

	// Statsistic members
	TIntFltH	LinkTriadH, CoMTriadH;
	TIntFltH MixInDegH, MixOutDegH, MixExtInDegH, MixExtOutDegH;
	TIntPrFltH InCompH, OutCompH;
	TIntFltH DomTmH;	

	TIntH TMPConnCntH;

private:
	// Set default values
	TStr DomBlackListFNm;
	int VSameCount;
	double VMinMultiple;
	double VMaxDifference;
	double VCoMThresh;
	int HiThresh;
	int LoThresh;
	int IsUseOnlyBlogDom;
	int IsUseOnlyMediaDom;

	void SetDefaultValues();
	void GetFeatureStrV();

public:
	TMemeTrend(PQuoteBs QtBs, TStr Pref, bool IsDataReady = false);

	// Initialize graphs
	void GetRawDomGraph();
	void GetDomLinkGraph();
	void GetDomCoMGraph();
	void GetDomMixGraph();
	void GetDomUNGraph();

	// Initialize pre-computed statistics
	void ComputeQtReportTime();
	void ComputeTriadH(TNodeEDatNet<TInt, TInt>& DomNet, TIntFltH& TriadH);
	void ComputeMixDegH();
	void ComputeCompH();
	void ComputeLinkJaccardSim(double& InSim, double& OutSim, int SrcId, int DstId);
	void ComputeCoMJaccardSim(double& InSim, double& OutSim, int SrcId, int DstId);

	// Feature computation
	void GetSubGraphFromDomNet(TVec<TFltV>& DomAdj, const TIntV& DomIdV, const TNodeEDatNet<TFlt, TFlt>& DomNet);
	void GetSubGraphFromDomNet(TVec<TFltV>& DomAdj, const TIntV& DomIdV, const TNodeEDatNet<TInt, TInt>& DomNet);
	double ComputeQtEdgeNum(const TVec<TFltV>& DomAdj);
	double ComputeQtTriadNum(const TVec<TFltV>& DomAdj);
	void ComputeQtGcc(const TVec<TFltV>& DomAdj, double& compNum, double& gccNodeNum, double& gccEdgeNum);
	void ComputeQtDeg(const TVec<TFltV>& DomAdj, double& maxDeg, double& d0NodeNum, double& d1NodeNum);	
	void ComputeQtInOutDeg(const TNodeEDatNet<TInt, TInt>& DomNet, const TIntV& DomIdV, double& AvgInDeg, double& AvgOutDeg);
	double ComputeQtDensity(const TVec<TFltV>& DomAdj);
	double ComputeQtClusterCoeff(const TVec<TFltV>& DomAdj);
	double ComputeQtMBRatio(const TIntV& DomIdV);
	double ComputeQtReportTime(const TIntV& DomIdV);
	double ComputeQtWordNum(const int QtN);
	void ComputeQtExt(double& ExtNodeNum, double& ExtEdgeNum, const TIntV& DomIdV, const TNodeEDatNet<TInt, TInt>& DomNet, const TIntFltH& TriadH);
	void ComputeQtMixDeg(double& MixInDeg, double& MixOutDeg, double& MixExtInDeg, double& MixExtOutDeg, const TIntV& DomIdV);
	double ComputeQtEntExcessDeg(const TVec<TFltV>& DomAdj);
	double ComputeTimeSpan(int QtN);
	void ComputeQtCompetitionCoeff(double& InCompCoeff, double& OutCompCoeff, const TIntV& DomIdV);

	// Feature vector computation
	void ComputeConnDomIdSet(TIntSet& ConnDomIdSet, const TIntV& DomIdV);
	void ComputeFeatureVector(TFltV& FeatureV, const int QtN, TIntV& DomIdV, int MaxN = -1);
	void ComputeAllFeatureVectors(int VBegDomNum, THash<TInt, TFltV>& FeatureVH);
	void ComputeAllFeatures();

	// Machine learning tasks
	void DumpClassificationTaskData(int VBegDomNum, bool IsFeatureReady = false);
	void DumpDifferentiationTaskData(const TIntPrV& QtPrV, int VBegDomNum, bool IsFeatureReady = false);
	void DumpNaiveBayesTaskData(int VBegDomNum);

	// Feature properties
	void PlotPopVSFeatureVal(int VBegDomNum, bool IsFeatureReady = false);
	void PlotFeatureValDistribution(int VBegDomNum, bool IsFeatureReady = false);

	// Feature evolution
	void EvoComputeCompCoeff(double& CpIn, double& CpOut, TIntV& DomIdV, int N);
	void EvoComputePDenCC(double& PDen, double& PCC, TIntV& DomIdV, int N);
	void EvoComputeCDenCC(double& CDen, double& CCC, TIntV& DomIdV, int N);
	void EvoComputeMixDeg(double& DegIn, double& DegOut, TIntV& DomIdV, int N);
	void PlotFeatureEvolution(TIntFltH& SumH, TIntFltH& SqrH, double TotCnt, TStr FNm, TStr Desc);
	void PlotFeatureEvolution(TIntFltH& HiSumH, TIntFltH& LoSumH, TStr FNm, TStr Desc);
	void FeatureEvolution(int VBegDomNum);

	// Meme trend
	void GetTrendQtPrV(TIntPrV& QtPrV, int MinUrls, int TopN, const TTmUnit& TmUnit = tmu1Hour, int TmCnt = 6);	
	void ComputeQtSig(int QtN, TQuote::TTmFltPrV& SmoothFqOtV, const TTmUnit& TmUnit, int TmCnt, TSecTm BegTime = TSecTm(0));
	void ComputeQtSig(TQuote& Qt, TQuote::TTmFltPrV& SmoothFqOtV, const TTmUnit& TmUnit, int TmCnt, TSecTm BegTime = TSecTm(0));
	void PlotTrendQtPrV(TIntPrV& QtPrV, TStr Pref, const TTmUnit& TmUnit, int TmCnt);
	void GetTmFltDataPr(TFltPrV& Data, TQuote::TTmFltPrV& SigV, const TTmUnit& TmUnit);
	void PlotClusterPair(TStr& OutFNm, int QtN1, int QtN2, TQuote::TTmFltPrV& SigV1, TQuote::TTmFltPrV& SigV2, const TTmUnit& TmUnit);
	void PlotTopClusterVariants(int TopNClust, int TopNVar, TStr Pref, const TTmUnit& TmUnit, int TmCnt);
	void DumpClusterPairHTML(TStr& FN, int QtN1, int QtN2, const TTmUnit& TmUnit, const int& TmCnt);
	void DumpTrendQt(TIntPrV& QtVarPrV, TStr OutFNm, bool SkipUrls);
	void DumpFeatureStats(const TIntPrV& QtPrV, const TTmUnit& TmUnit, int TmCnt);

	// Graph properties
	void CoMentionLinkG();
	void GetAvgDist(PUNGraph& G, double& AvgDist, double& D);
	void RewireCoMGraph();
	void PrintDomNetInfo(const TNodeEDatNet<TFlt, TFlt>& DomNet);
	void PrintDomNetInfo(const TNodeEDatNet<TInt, TInt>& DomNet);

	// Utility function 
	TStr GetDomNmFromUrl(TStr UrlStr);
	void GetQtDomIdV(TIntV& DomIdV, int QtN, int MaxN = -1, const TTmUnit& TmUnit = tmu1Hour, int TmCnt = 5);
	void DumpFreqConnector();
	void DumpDomainByLinkDeg();	

	// Test function
	void GetDuplicateDoms();
	void DomainPrediction(TIntPrV& QtVarPrV, int VBegDomNum);
	void InvestigateBadQuote(int VBegDomNum);

	void TESTDataSetStats();
};


#endif
